{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english_news = glob('/home/husein/google-translate-english-news/*.requested')\n",
    "len(english_news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english_texts = glob('/home/husein/google-translate-english-texts/*.requested')\n",
    "len(english_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "malay_news = glob('/home/husein/google-translate-malay-news/*.requested')\n",
    "len(malay_news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hansard = glob('/home/husein/google-translate-malaysia-hansard/*.requested')\n",
    "len(hansard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ens = [english_news, english_texts]\n",
    "mss = [malay_news, hansard]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2496675it [00:13, 190186.52it/s]\n",
      "1683631it [00:09, 183901.04it/s]\n",
      "958405it [00:08, 114555.38it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('train.json', 'w') as fopen_jsonl:\n",
    "    for f in ['processed-ms-id.jsonl', 'processed-ms-jw.jsonl']:\n",
    "        with open(f) as fopen:\n",
    "            for l in tqdm(fopen):\n",
    "                l = json.loads(l)\n",
    "                left = l['left']\n",
    "                right = l['right']\n",
    "                if len(left) and len(left.split()) < 1200 and len(right) and len(right.split()) < 1200:\n",
    "                    d = {\"translation\": {\"src\": right, \"tgt\": left, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('process-bjn-Latn.jsonl') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            l = json.loads(l)\n",
    "            left = l['left']\n",
    "            chatgpt = l['chatgpt']\n",
    "            google = l['google']\n",
    "            \n",
    "            if len(left) and len(left.split()) < 1200 and len(google) and len(google.split()) < 1200:\n",
    "                d = {\"translation\": {\"src\": left, \"tgt\": google, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "            if len(chatgpt) and len(chatgpt.split()) < 1200 and len(google) and len(google.split()) < 1200:\n",
    "                d = {\"translation\": {\"src\": chatgpt, \"tgt\": google, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "691083it [00:12, 55674.51it/s]\n",
      "18080it [00:00, 60299.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42516it [00:00, 61139.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "67167it [00:01, 60834.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "79307it [00:01, 60012.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "103208it [00:01, 59336.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "115164it [00:01, 59569.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "132991it [00:02, 59148.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "141912it [00:02, 59421.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3643it [00:00, 36422.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19042it [00:00, 38406.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "34408it [00:00, 38336.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "57094it [00:01, 37327.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'list' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "72372it [00:01, 37670.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "87356it [00:02, 36873.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'list' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "106316it [00:02, 37872.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "113884it [00:03, 37389.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'list' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "128925it [00:03, 36857.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "139903it [00:03, 36125.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "154369it [00:04, 36007.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "161591it [00:04, 35954.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "168851it [00:04, 35942.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "179646it [00:04, 35624.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'list' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "186857it [00:05, 35241.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "197405it [00:05, 35032.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "214961it [00:05, 33675.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'list' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "225377it [00:06, 34074.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'list' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "232456it [00:06, 34719.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "253566it [00:07, 34450.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "264242it [00:07, 34840.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "274907it [00:07, 35009.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "282118it [00:07, 35557.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'list' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "292609it [00:08, 33903.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "299549it [00:08, 33937.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "310216it [00:08, 34695.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "316998it [00:08, 32271.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "327423it [00:09, 33965.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "333758it [00:09, 35439.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'dict' object has no attribute 'split'\n",
      "'dict' object has no attribute 'split'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    for f in ['processed-twitter.jsonl', 'processed-facebook.jsonl', 'processed-iium-confession.jsonl']:\n",
    "        with open(f) as fopen:\n",
    "            for l in tqdm(fopen):\n",
    "                try:\n",
    "                    l = json.loads(l)\n",
    "                    left = l['left']\n",
    "                    en = l['en']\n",
    "                    ms = l['ms']\n",
    "\n",
    "                    if len(left) and len(left.split()) < 1200 and len(en) and len(en.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": left, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": left, 'prefix': 'terjemah ke pasar Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                    if len(left) and len(left.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": left, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": left, 'prefix': 'terjemah ke pasar Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                    if len(ms) and len(ms.split()) < 1200 and len(en) and len(en.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                except Exception as e:\n",
    "                    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75777it [00:03, 19451.18it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('gpt4all-code.jsonl') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            try:\n",
    "                l = json.loads(l)\n",
    "                left = l['prompt'][0]\n",
    "                right = l['prompt'][1]\n",
    "                \n",
    "                if len(left) and len(right):\n",
    "                    d = {\"translation\": {\"src\": left, \"tgt\": right, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": right, \"tgt\": left, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "                left = l['response'][0]\n",
    "                right = l['response'][1]\n",
    "                \n",
    "                if len(left) and len(right):\n",
    "                    d = {\"translation\": {\"src\": left, \"tgt\": right, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": right, \"tgt\": left, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18837it [00:00, 114086.02it/s]\n",
      "100000it [00:00, 119631.46it/s]\n",
      "100000it [00:00, 123175.89it/s]\n",
      "100000it [00:00, 121132.72it/s]\n",
      "100000it [00:00, 123827.62it/s]\n",
      "100000it [00:00, 124922.49it/s]\n",
      "100000it [00:00, 123734.03it/s]\n",
      "100000it [00:00, 121127.96it/s]\n",
      "100000it [00:00, 124583.90it/s]\n",
      "100000it [00:00, 124019.00it/s]\n",
      "100000it [00:01, 71865.05it/s]\n",
      "100000it [00:01, 76983.28it/s]\n",
      "100000it [00:00, 114809.33it/s]\n",
      "100000it [00:00, 123640.15it/s]\n",
      "100000it [00:00, 126191.58it/s]\n",
      "100000it [00:00, 125383.43it/s]\n",
      "100000it [00:00, 127454.72it/s]\n",
      "100000it [00:00, 126554.86it/s]\n",
      "100000it [00:00, 124706.14it/s]\n",
      "100000it [00:00, 127133.45it/s]\n",
      "100000it [00:00, 126360.30it/s]\n",
      "100000it [00:00, 126934.84it/s]\n",
      "100000it [00:00, 127029.64it/s]\n",
      "100000it [00:00, 126294.55it/s]\n",
      "100000it [00:00, 121686.90it/s]\n",
      "100000it [00:00, 124045.67it/s]\n",
      "100000it [00:00, 125460.17it/s]\n",
      "100000it [00:00, 125644.70it/s]\n",
      "100000it [00:00, 123199.69it/s]\n",
      "100000it [00:00, 125622.01it/s]\n",
      "99999it [00:00, 124880.85it/s]\n",
      "100000it [00:00, 125708.15it/s]\n",
      "99990it [00:01, 99554.50it/s] \n",
      "99997it [00:00, 100619.77it/s]\n",
      "99997it [00:01, 99283.44it/s]\n",
      "99996it [00:01, 97635.10it/s]\n",
      "99998it [00:01, 99471.48it/s] \n",
      "99993it [00:01, 99072.52it/s]\n",
      "7616it [00:00, 96064.70it/s]\n",
      "99995it [00:01, 97504.99it/s]\n",
      "99995it [00:01, 97965.44it/s]\n",
      "99997it [00:01, 99257.38it/s] \n",
      "99992it [00:01, 98900.91it/s]\n",
      "99994it [00:01, 99488.08it/s] \n",
      "99993it [00:01, 91069.51it/s]\n",
      "99988it [00:01, 93102.20it/s]\n",
      "99993it [00:01, 93376.67it/s]\n",
      "99999it [00:01, 96650.73it/s]\n",
      "99992it [00:01, 97275.56it/s]\n",
      "99994it [00:01, 95986.81it/s]\n",
      "99996it [00:01, 99071.21it/s]\n",
      "99999it [00:01, 98531.22it/s]\n",
      "99994it [00:01, 99280.27it/s] \n",
      "99996it [00:01, 99075.22it/s] \n",
      "99993it [00:01, 99440.92it/s] \n",
      "99995it [00:01, 97705.63it/s]\n",
      "99994it [00:01, 99569.92it/s] \n",
      "99990it [00:01, 94398.80it/s]\n",
      "99996it [00:01, 97594.57it/s]\n",
      "99993it [00:00, 100643.68it/s]\n",
      "99996it [00:01, 98459.84it/s]\n",
      "99991it [00:01, 97625.81it/s]\n",
      "99996it [00:00, 100348.89it/s]\n",
      "99995it [00:01, 99254.41it/s] \n",
      "99997it [00:01, 98382.74it/s]\n",
      "99992it [00:01, 98677.22it/s] \n",
      "99994it [00:01, 98442.11it/s]\n",
      "99998it [00:00, 100747.67it/s]\n",
      "99992it [00:01, 97801.10it/s] \n",
      "99999it [00:00, 100254.95it/s]\n",
      "99996it [00:00, 100172.90it/s]\n",
      "87609it [00:00, 129759.54it/s]\n",
      "100000it [00:00, 120290.27it/s]\n",
      "100000it [00:00, 122066.33it/s]\n",
      "100000it [00:00, 118009.59it/s]\n",
      "99897it [00:00, 121799.41it/s]\n",
      "99999it [00:00, 119405.11it/s]\n",
      "99992it [00:00, 116235.79it/s]\n",
      "99994it [00:00, 126588.14it/s]\n",
      "99997it [00:01, 98520.06it/s] \n",
      "99998it [00:00, 119724.78it/s]\n",
      "100000it [00:00, 119968.91it/s]\n",
      "99989it [00:00, 119627.31it/s]\n",
      "100000it [00:00, 117726.16it/s]\n",
      "99994it [00:00, 122769.59it/s]\n",
      "100000it [00:00, 118267.51it/s]\n",
      "100000it [00:00, 118313.31it/s]\n",
      "100000it [00:00, 119354.76it/s]\n",
      "100000it [00:00, 122129.20it/s]\n",
      "100000it [00:00, 126426.84it/s]\n",
      "100000it [00:00, 118343.99it/s]\n",
      "100000it [00:00, 122986.95it/s]\n",
      "99995it [00:00, 125709.89it/s]\n",
      "43098it [00:03, 13603.83it/s]\n",
      "36329it [00:02, 13740.32it/s]\n",
      "42876it [00:03, 13794.85it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    for fs in ens:\n",
    "        for f in fs:\n",
    "            with open(f) as fopen:\n",
    "                for l in tqdm(fopen):\n",
    "                    data = json.loads(l)\n",
    "                    en = data['src']\n",
    "                    ms = data['r']['result']\n",
    "\n",
    "                    if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                        \n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                        \n",
    "    for fs in mss:\n",
    "        for f in fs:\n",
    "            with open(f) as fopen:\n",
    "                for l in tqdm(fopen):\n",
    "                    data = json.loads(l)\n",
    "                    if isinstance(data['src'], dict):\n",
    "                        s = data['src']['cleaned']\n",
    "                    else:\n",
    "                        s = data['src']\n",
    "                    ms = s\n",
    "                    en = data['r']['result']\n",
    "\n",
    "                    if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                        \n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                        \n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31872275 train.json\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"Terminal 1 KKIA dilengkapi dengan 64 konter check-in, 12 garbarata dan dapat menampung 3.200 penumpang sekaligus.\", \"tgt\": \"Terminal 1 KKIA dilengkapi kemudahan 64 kaunter daftar masuk, 12 aero bridge selain mampu menampung 3,200 penumpang dalam satu masa.\", \"prefix\": \"terjemah ke Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"\\\"Oposisi hanya berani bicara besar-besaran, seolah terlihat lebih baik dari berita,\\\" ujarnya.\", \"tgt\": \"\\\"Pembangkang hanya berani cakap besar, seolah-olah indah rupa daripada berita,\\\" katanya.\", \"prefix\": \"terjemah ke Melayu: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!head -n 2 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"Puan Hajah Zuraida Kamaruddin [Ampang]: A little more, Deputy Minister. I understand that there is this program and so on, but how is the monitoring done so that these courses have a certain kind of upgrading, with permission. I see these courses are flat. From before until last year until now the same. So, there is no increase in terms of modules and in terms of teaching staff. So, that's what we think our country wants to become a developed country, women want to be involved in the economy and so on. Our children should be given quality care. This means that we have to upgrade the teachers who teach to degree holders with some qualifications that are specific to child care. Datuk Heng Seai Kie: Yes. Dear Ampang, thank you for the suggestions. It is true that the quality of caregivers or educators in nurseries or with the permission of kindergartens is important because they are the ones who will take care of and educate our children. I will take note of this. Yang Berhormat Petaling Jaya Selatan has touched on the issue of the Selangor Spastic Association and Kuala Lumpur Region where there are parents who complain that the special allowance for disabled children is only given as much as RM100 and another RM50 for PSK management. For your information, in 2011, JKM has already provided a special allowance for disabled children amounting to RM394,200 for 219 children in the Selangor Spastic Association and Kuala Lumpur Region. JKM has an officer who will monitor the granting of special allowances to disabled children and ensure that RM150 per month is received by each child. However, any other fees charged by the association are based on the agreement of parents and the association's committee. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Request an explanation Honorable Deputy Minister. What happened at the Children's Spastic Association of Selangor and Kuala Lumpur Region did exist. The association took the RM50 and only RM100 went into the account of the disabled children. Then I don't know if this is wrong or not. If it is said that RM150 is entitled to be received by disabled children, I hope the ministry can take immediate action because this has been happening for a long time. So, RM50 that the parents of the disabled did not agree with but they took it anyway. Thank you. Datuk Heng Seai Kie: Thank you for the additional question. Actually, as I mentioned, the RM150 allowance we pay goes to the disabled children. If they pay RM50 out of it to the operator which is the association concerned, that is based on a mutual agreement between the parents and the committee. If... Tuan Hee Loy Sian [Petaling Jaya Selatan]: I agree Honorable Deputy Minister. Datuk Heng Seai Kie: If this is something beyond the meaning of not getting the consent of the parents, then we call for the parents to write or complain to the central JKM. We will take action. Yang Berhormat Petaling Jaya Selatan has also already asked about the abandoned carer who receives RM300 whether the children who enter PDK are eligible to receive an allowance of RM150. For your information, at this time the ministry does not prevent a family from receiving more than one aid if eligible. In connection with that, children who enter PDK can receive a PDK allowance of RM150 per month even if their guardians receive RM300 per month for homeless disabled persons. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Honorable Deputy Minister. Datuk Heng Seai Kie: So, if there is a case... Tuan Hee Loy Sian [Petaling Jaya Selatan]: That has already happened, there is also a case. Datuk Heng Seai Kie: If there is a case Your Honor can refer the case to me. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Okay, thank you. Datuk Heng Seai Kie: Your Honor also asked about the EPC which is the Disabled Employees Allowance so that it should be increased from RM300 to a higher rate.\", \"tgt\": \"Puan Hajah Zuraida Kamaruddin [Ampang]: Sedikit lagi ya Timbalan Menteri. Saya faham memang ada program ini dan sebagainya tetapi bagaimana pemantauan itu agar kursus-kursus ini ada certain kind of upgrading ,dengan izin. Saya lihat kursus-kursus ini mendatar. Daripada dahulu sampai tahun lepas sampai sekarang yang sama. Jadi, tidak ada peningkatan dari segi modul dan dari segi tenaga pengajar. Jadi, itu yang kita rasa negara kita hendak jadi negara maju, orang perempuan pun hendak terlibat dalam ekonomi dan sebagainya. Anak-anak kita ini harus diberikan satu penjagaan yang bermutu. Maksudnya guru-guru yang mengajar itu kita kena upgrade degree holders with some qualifications yang spesifik untuk penjagaan anak-anak. Datuk Heng Seai Kie: Ya. Yang Berhormat Ampang terima kasih atas cadangan yang dikemukakan. Memang betul mutu ataupun kualiti pengasuh ataupun pendidik di taska ataupun dengan izin kindergartens adalah penting kerana merekalah yang akan menjaga dan mendidik anak-anak kita. Perkara ini saya akan ambil maklum. Yang Berhormat Petaling Jaya Selatan telah menyentuh tentang isu Persatuan Spastik Selangor dan Wilayah Kuala Lumpur di mana terdapat ibu bapa yang mengadu elaun khas kanak-kanak OKU hanya diberikan sebanyak RM100 dan RM50 lagi untuk pengurusan PSK. Untuk makluman Yang Berhormat, pada tahun 2011, JKM telah pun menyediakan elaun khas kanak-kanak OKU sebanyak RM394,200 untuk 219 orang kanak- kanak di Persatuan Spastik Selangor dan Wilayah Kuala Lumpur. JKM mempunyai pegawai yang akan memantau pemberian elaun khas kanak-kanak OKU dan memastikan supaya RM150 sebulan diterima oleh setiap kanak-kanak tersebut. Walau bagaimanapun, sebarang bayaran lain yang dikenakan oleh persatuan berkenaan adalah berdasarkan persetujuan ibu bapa dan jawatankuasa persatuan tersebut. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Minta penjelasan Yang Berhormat Timbalan Menteri. Apa yang berlaku di Persatuan Spastik Kanak-kanak Selangor dan Wilayah Kuala Lumpur memang wujud persatuan itu ambil RM50 itu dan hanya masuk RM100 kepada akaun kanak-kanak OKU. Kalau begitu saya tidak tahu ini salah atau tidak. Kalau kata RM150 itu berhak diterima oleh kanak-kanak OKU, saya harap kementerian dapat ambil tindakan segera kerana hal ini sudah lama berlaku. So, RM50 yang tidak setuju oleh ibu bapa OKU tetapi mereka ambil juga. Terima kasih. Datuk Heng Seai Kie: Terima kasih Yang Berhormat atas soalan tambahan. Sebenarnya, sepertimana yang saya nyatakan elaun RM150 ini kita bayar tuju kepada kanak-kanak OKU tersebut. Sekiranya mereka membayar RM50 daripadanya kepada pengendali iaitu persatuan yang berkenaan, itu adalah berdasarkan kepada persetujuan bersama antara ibu bapa dan jawatankuasa. Sekiranya... Tuan Hee Loy Sian [Petaling Jaya Selatan]: Saya setuju Yang Berhormat Timbalan Menteri. Datuk Heng Seai Kie: Sekiranya ini adalah satu di luar maksudnya tidak mendapat persetujuan ibu bapa maka kita menyeru agar ibu bapa itu bolehlah tulis ataupun mengadu kepada JKM pusat. Kita akan ambil tindakan. Yang Berhormat Petaling Jaya Selatan juga tadi telah pun bertanya tentang penjaga terlantar yang terima RM300 sama ada anak-anak yang masuk PDK layak untuk terima elaun sebanyak RM150. Untuk makluman Yang Berhormat, pada masa ini kementerian tidak menghalang satu keluarga untuk menerima lebih daripada satu bantuan sekiranya layak. Sehubungan dengan itu anak-anak yang masuk PDK boleh menerima elaun PDK sebanyak RM150 sebulan walaupun penjaganya menerima bantuan OKU terlantar sebanyak RM300 sebulan. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Yang Berhormat Timbalan Menteri. Datuk Heng Seai Kie: Jadi, kalau ada kes... Tuan Hee Loy Sian [Petaling Jaya Selatan]: Itu sudah berlaku, ada kes juga. Datuk Heng Seai Kie: Kalau ada kes Yang Berhormat bolehlah rujukkan kes itu kepada saya ya. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Okey, terima kasih. Datuk Heng Seai Kie: Yang Berhormat juga tadi bertanya tentang EPC iaitu Elaun Pekerja Cacat supaya ia harus dinaikkan daripada RM300 kepada kadar yang lebih tinggi.\", \"prefix\": \"terjemah ke Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Puan Hajah Zuraida Kamaruddin [Ampang]: Sedikit lagi ya Timbalan Menteri. Saya faham memang ada program ini dan sebagainya tetapi bagaimana pemantauan itu agar kursus-kursus ini ada certain kind of upgrading ,dengan izin. Saya lihat kursus-kursus ini mendatar. Daripada dahulu sampai tahun lepas sampai sekarang yang sama. Jadi, tidak ada peningkatan dari segi modul dan dari segi tenaga pengajar. Jadi, itu yang kita rasa negara kita hendak jadi negara maju, orang perempuan pun hendak terlibat dalam ekonomi dan sebagainya. Anak-anak kita ini harus diberikan satu penjagaan yang bermutu. Maksudnya guru-guru yang mengajar itu kita kena upgrade degree holders with some qualifications yang spesifik untuk penjagaan anak-anak. Datuk Heng Seai Kie: Ya. Yang Berhormat Ampang terima kasih atas cadangan yang dikemukakan. Memang betul mutu ataupun kualiti pengasuh ataupun pendidik di taska ataupun dengan izin kindergartens adalah penting kerana merekalah yang akan menjaga dan mendidik anak-anak kita. Perkara ini saya akan ambil maklum. Yang Berhormat Petaling Jaya Selatan telah menyentuh tentang isu Persatuan Spastik Selangor dan Wilayah Kuala Lumpur di mana terdapat ibu bapa yang mengadu elaun khas kanak-kanak OKU hanya diberikan sebanyak RM100 dan RM50 lagi untuk pengurusan PSK. Untuk makluman Yang Berhormat, pada tahun 2011, JKM telah pun menyediakan elaun khas kanak-kanak OKU sebanyak RM394,200 untuk 219 orang kanak- kanak di Persatuan Spastik Selangor dan Wilayah Kuala Lumpur. JKM mempunyai pegawai yang akan memantau pemberian elaun khas kanak-kanak OKU dan memastikan supaya RM150 sebulan diterima oleh setiap kanak-kanak tersebut. Walau bagaimanapun, sebarang bayaran lain yang dikenakan oleh persatuan berkenaan adalah berdasarkan persetujuan ibu bapa dan jawatankuasa persatuan tersebut. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Minta penjelasan Yang Berhormat Timbalan Menteri. Apa yang berlaku di Persatuan Spastik Kanak-kanak Selangor dan Wilayah Kuala Lumpur memang wujud persatuan itu ambil RM50 itu dan hanya masuk RM100 kepada akaun kanak-kanak OKU. Kalau begitu saya tidak tahu ini salah atau tidak. Kalau kata RM150 itu berhak diterima oleh kanak-kanak OKU, saya harap kementerian dapat ambil tindakan segera kerana hal ini sudah lama berlaku. So, RM50 yang tidak setuju oleh ibu bapa OKU tetapi mereka ambil juga. Terima kasih. Datuk Heng Seai Kie: Terima kasih Yang Berhormat atas soalan tambahan. Sebenarnya, sepertimana yang saya nyatakan elaun RM150 ini kita bayar tuju kepada kanak-kanak OKU tersebut. Sekiranya mereka membayar RM50 daripadanya kepada pengendali iaitu persatuan yang berkenaan, itu adalah berdasarkan kepada persetujuan bersama antara ibu bapa dan jawatankuasa. Sekiranya... Tuan Hee Loy Sian [Petaling Jaya Selatan]: Saya setuju Yang Berhormat Timbalan Menteri. Datuk Heng Seai Kie: Sekiranya ini adalah satu di luar maksudnya tidak mendapat persetujuan ibu bapa maka kita menyeru agar ibu bapa itu bolehlah tulis ataupun mengadu kepada JKM pusat. Kita akan ambil tindakan. Yang Berhormat Petaling Jaya Selatan juga tadi telah pun bertanya tentang penjaga terlantar yang terima RM300 sama ada anak-anak yang masuk PDK layak untuk terima elaun sebanyak RM150. Untuk makluman Yang Berhormat, pada masa ini kementerian tidak menghalang satu keluarga untuk menerima lebih daripada satu bantuan sekiranya layak. Sehubungan dengan itu anak-anak yang masuk PDK boleh menerima elaun PDK sebanyak RM150 sebulan walaupun penjaganya menerima bantuan OKU terlantar sebanyak RM300 sebulan. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Yang Berhormat Timbalan Menteri. Datuk Heng Seai Kie: Jadi, kalau ada kes... Tuan Hee Loy Sian [Petaling Jaya Selatan]: Itu sudah berlaku, ada kes juga. Datuk Heng Seai Kie: Kalau ada kes Yang Berhormat bolehlah rujukkan kes itu kepada saya ya. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Okey, terima kasih. Datuk Heng Seai Kie: Yang Berhormat juga tadi bertanya tentang EPC iaitu Elaun Pekerja Cacat supaya ia harus dinaikkan daripada RM300 kepada kadar yang lebih tinggi.\", \"tgt\": \"Puan Hajah Zuraida Kamaruddin [Ampang]: A little more, Deputy Minister. I understand that there is this program and so on, but how is the monitoring done so that these courses have a certain kind of upgrading, with permission. I see these courses are flat. From before until last year until now the same. So, there is no increase in terms of modules and in terms of teaching staff. So, that's what we think our country wants to become a developed country, women want to be involved in the economy and so on. Our children should be given quality care. This means that we have to upgrade the teachers who teach to degree holders with some qualifications that are specific to child care. Datuk Heng Seai Kie: Yes. Dear Ampang, thank you for the suggestions. It is true that the quality of caregivers or educators in nurseries or with the permission of kindergartens is important because they are the ones who will take care of and educate our children. I will take note of this. Yang Berhormat Petaling Jaya Selatan has touched on the issue of the Selangor Spastic Association and Kuala Lumpur Region where there are parents who complain that the special allowance for disabled children is only given as much as RM100 and another RM50 for PSK management. For your information, in 2011, JKM has already provided a special allowance for disabled children amounting to RM394,200 for 219 children in the Selangor Spastic Association and Kuala Lumpur Region. JKM has an officer who will monitor the granting of special allowances to disabled children and ensure that RM150 per month is received by each child. However, any other fees charged by the association are based on the agreement of parents and the association's committee. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Request an explanation Honorable Deputy Minister. What happened at the Children's Spastic Association of Selangor and Kuala Lumpur Region did exist. The association took the RM50 and only RM100 went into the account of the disabled children. Then I don't know if this is wrong or not. If it is said that RM150 is entitled to be received by disabled children, I hope the ministry can take immediate action because this has been happening for a long time. So, RM50 that the parents of the disabled did not agree with but they took it anyway. Thank you. Datuk Heng Seai Kie: Thank you for the additional question. Actually, as I mentioned, the RM150 allowance we pay goes to the disabled children. If they pay RM50 out of it to the operator which is the association concerned, that is based on a mutual agreement between the parents and the committee. If... Tuan Hee Loy Sian [Petaling Jaya Selatan]: I agree Honorable Deputy Minister. Datuk Heng Seai Kie: If this is something beyond the meaning of not getting the consent of the parents, then we call for the parents to write or complain to the central JKM. We will take action. Yang Berhormat Petaling Jaya Selatan has also already asked about the abandoned carer who receives RM300 whether the children who enter PDK are eligible to receive an allowance of RM150. For your information, at this time the ministry does not prevent a family from receiving more than one aid if eligible. In connection with that, children who enter PDK can receive a PDK allowance of RM150 per month even if their guardians receive RM300 per month for homeless disabled persons. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Honorable Deputy Minister. Datuk Heng Seai Kie: So, if there is a case... Tuan Hee Loy Sian [Petaling Jaya Selatan]: That has already happened, there is also a case. Datuk Heng Seai Kie: If there is a case Your Honor can refer the case to me. Tuan Hee Loy Sian [Petaling Jaya Selatan]: Okay, thank you. Datuk Heng Seai Kie: Your Honor also asked about the EPC which is the Disabled Employees Allowance so that it should be increased from RM300 to a higher rate.\", \"prefix\": \"terjemah ke Inggeris: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 2 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "!shuf train.json > shuffled-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
